{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import random\n",
    "import math\n",
    "import re\n",
    "import time\n",
    "import numpy as np\n",
    "import cv2\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import h5py \n",
    "import datetime\n",
    "\n",
    "# Root directory of the project\n",
    "path = '/mnt/sh_flex_storage/malu/venv/CHIPS_MRCNN'\n",
    "ROOT_DIR = os.path.abspath(path)\n",
    "\n",
    "# Import Mask RCNN\n",
    "sys.path.append(ROOT_DIR)  # To find local version of the library\n",
    "from mrcnn_chips.config import Config\n",
    "from mrcnn_chips import utils\n",
    "import mrcnn_chips.model as modellib\n",
    "from mrcnn_chips import visualize\n",
    "from mrcnn_chips.model import log\n",
    "\n",
    "%matplotlib inline \n",
    "\n",
    "# Directory to save logs and trained model\n",
    "MODEL_DIR = os.path.join(ROOT_DIR, \"logs_six\")\n",
    "\n",
    "# Local path to trained weights file\n",
    "COCO_MODEL_PATH = os.path.join(ROOT_DIR, \"mask_rcnn_coco.h5\")\n",
    "# Download COCO trained weights from Releases if needed\n",
    "if not os.path.exists(COCO_MODEL_PATH):\n",
    "    utils.download_trained_weights(COCO_MODEL_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROOT_DIR: /mnt/sh_flex_storage/malu/venv/CHIPS_MRCNN\n",
      "MODEL_DIR: /mnt/sh_flex_storage/malu/venv/CHIPS_MRCNN/logs_six\n",
      "COCO_MODEL_PATH: /mnt/sh_flex_storage/malu/venv/CHIPS_MRCNN/mask_rcnn_coco.h5\n"
     ]
    }
   ],
   "source": [
    "print('ROOT_DIR:',ROOT_DIR)\n",
    "print('MODEL_DIR:',MODEL_DIR)\n",
    "print('COCO_MODEL_PATH:',COCO_MODEL_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_path = 'new_train.h5'\n",
    "test_path = 'new_test.h5'\n",
    "train_set = h5py.File(train_path,'r')\n",
    "test_set = h5py.File(test_path,'r')\n",
    "\n",
    "train_images = train_set['input']\n",
    "test_images = test_set['input']\n",
    "train_masks = train_set['output']\n",
    "test_masks = test_set['output']\n",
    "\n",
    "del train_set\n",
    "del test_set\n",
    "NUM_TRAIN = train_images.shape[0] # 224*224\n",
    "NUM_TEST = test_images.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Configurations:\n",
      "BACKBONE                       resnet50\n",
      "BACKBONE_STRIDES               [4, 8, 16, 32, 64]\n",
      "BATCH_SIZE                     8\n",
      "BBOX_STD_DEV                   [0.1 0.1 0.2 0.2]\n",
      "COMPUTE_BACKBONE_SHAPE         None\n",
      "DETECTION_MAX_INSTANCES        100\n",
      "DETECTION_MIN_CONFIDENCE       0.7\n",
      "DETECTION_NMS_THRESHOLD        0.3\n",
      "FPN_CLASSIF_FC_LAYERS_SIZE     1024\n",
      "GPU_COUNT                      1\n",
      "GRADIENT_CLIP_NORM             5.0\n",
      "IMAGES_PER_GPU                 8\n",
      "IMAGE_MAX_DIM                  256\n",
      "IMAGE_META_SIZE                16\n",
      "IMAGE_MIN_DIM                  256\n",
      "IMAGE_MIN_SCALE                0\n",
      "IMAGE_RESIZE_MODE              square\n",
      "IMAGE_SHAPE                    [256 256   6]\n",
      "LEARNING_MOMENTUM              0.9\n",
      "LEARNING_RATE                  0.001\n",
      "LOSS_WEIGHTS                   {'mrcnn_class_loss': 1.0, 'rpn_class_loss': 1.0, 'mrcnn_bbox_loss': 1.0, 'rpn_bbox_loss': 1.0, 'mrcnn_mask_loss': 1.0}\n",
      "MASK_POOL_SIZE                 14\n",
      "MASK_SHAPE                     [28, 28]\n",
      "MAX_GT_INSTANCES               100\n",
      "MEAN_PIXEL                     [123.7 116.8 103.9]\n",
      "MINI_MASK_SHAPE                (28, 28)\n",
      "NAME                           chips\n",
      "NUM_CLASSES                    4\n",
      "POOL_SIZE                      7\n",
      "POST_NMS_ROIS_INFERENCE        1000\n",
      "POST_NMS_ROIS_TRAINING         2000\n",
      "ROI_POSITIVE_RATIO             0.16\n",
      "RPN_ANCHOR_RATIOS              [0.5, 1, 2]\n",
      "RPN_ANCHOR_SCALES              (4, 8, 16, 32, 64)\n",
      "RPN_ANCHOR_STRIDE              1\n",
      "RPN_BBOX_STD_DEV               [0.1 0.1 0.2 0.2]\n",
      "RPN_NMS_THRESHOLD              0.7\n",
      "RPN_TRAIN_ANCHORS_PER_IMAGE    256\n",
      "STEPS_PER_EPOCH                100\n",
      "TOP_DOWN_PYRAMID_SIZE          256\n",
      "TRAIN_BN                       False\n",
      "TRAIN_ROIS_PER_IMAGE           16\n",
      "USE_MINI_MASK                  True\n",
      "USE_RPN_ROIS                   True\n",
      "VALIDATION_STEPS               5\n",
      "WEIGHT_DECAY                   0.0001\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\nnot apply in network yet\\nROI_POSITIVE_RATIO = 0.33 # before 0.16\\nSTEPS_PER_EPOCH = 200\\n'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class ChipsConfig(Config):\n",
    "    \"\"\"Configuration for training on the toy shapes dataset.\n",
    "    Derives from the base Config class and overrides values specific\n",
    "    to the toy shapes dataset.\n",
    "    \"\"\"\n",
    "    # Give the configuration a recognizable name\n",
    "    NAME = \"chips\"\n",
    "    \n",
    "    #CH\n",
    "    BACKBONE = \"resnet50\"\n",
    "    MINI_MASK_SHAPE = (28,28)\n",
    "    DETECTION_MAX_INSTANCES = 100\n",
    "    ROI_POSITIVE_RATIO = 0.16\n",
    "    # USE_MINI_MASK = False # avoid warn of anti-aliasing \n",
    "    \n",
    "    # TRY CH\n",
    "    POOL_SIZE = 7\n",
    "    MASK_POOL_SIZE = 14\n",
    "    \n",
    "    # Train on 1 GPU and 8 images per GPU. We can put multiple images on each\n",
    "    # GPU because the images are small. Batch size is 8 (GPUs * images/GPU).\n",
    "    GPU_COUNT = 1 # CH\n",
    "    IMAGES_PER_GPU = 8\n",
    "\n",
    "    # Number of classes (including background)\n",
    "    NUM_CLASSES = 1 + 3  # background + 3 shapes\n",
    "\n",
    "    # Use small images for faster training. Set the limits of the small side\n",
    "    # the large side, and that determines the image shape.\n",
    "    IMAGE_MIN_DIM = 256\n",
    "    IMAGE_MAX_DIM = 256\n",
    "\n",
    "    # Use smaller anchors because our image and objects are small\n",
    "    RPN_ANCHOR_SCALES = (4, 8, 16, 32, 64)  # anchor side in pixels # CH\n",
    "\n",
    "    # Reduce training ROIs per image because the images are small and have\n",
    "    # few objects. Aim to allow ROI sampling to pick 33% positive ROIs.\n",
    "    TRAIN_ROIS_PER_IMAGE = 16 # CH \n",
    "\n",
    "    # Use a small epoch since the data is simple\n",
    "    STEPS_PER_EPOCH = 100\n",
    "        \n",
    "    # use small validation steps since the epoch is small\n",
    "    VALIDATION_STEPS = 5\n",
    "    \n",
    "    \n",
    "    \n",
    "config = ChipsConfig()\n",
    "config.display()\n",
    "\n",
    "'''\n",
    "not apply in network yet\n",
    "ROI_POSITIVE_RATIO = 0.33 # before 0.16\n",
    "STEPS_PER_EPOCH = 200\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Notebook Preferences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def get_ax(rows=1, cols=1, size=8):\n",
    "    _, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows))\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class ChipsDataset(utils.Dataset):\n",
    "    def __init__(self, class_map=None):\n",
    "        #new_append \n",
    "        self.num_null_mask = 0\n",
    "        self.all_flaw_size = []\n",
    "        self.subset = \"\"\n",
    "        #original\n",
    "        self._image_ids = []\n",
    "        self.image_info = []\n",
    "        # Background is always the first class\n",
    "        self.class_info = [{\"source\": \"\", \"id\": 0, \"name\": \"BG\"}]\n",
    "        self.source_class_ids = {}\n",
    "\n",
    "    def load_chips(self, count,subset):\n",
    "        assert subset in [\"train\",\"test\"]\n",
    "        \n",
    "        self.subset = subset\n",
    "        \n",
    "        # Add classes\n",
    "        self.add_class(\"chips\", 1, \"bump\")\n",
    "        self.add_class(\"chips\", 2, \"dent\")\n",
    "        self.add_class(\"chips\", 3, \"dot\")\n",
    "\n",
    "        # Add images\n",
    "        # Images are generated on the fly in load_image().\n",
    "        for i in range(count):\n",
    "            chips = self.mask_prepare(i)\n",
    "            self.add_image(\"chips\", image_id=i, path=None,chips=chips)\n",
    "            \n",
    "    \n",
    "    def mask_prepare(self,i,return_mask=False):\n",
    "        if self.subset == \"train\":\n",
    "            mask_full = train_masks[i,:].reshape((4,224,224))\n",
    "        else :\n",
    "            mask_full = test_masks[i,:].reshape((4,224,224))\n",
    "        \n",
    "        bump,dent,dot = mask_full[1,:,:],mask_full[2,:,:],mask_full[3,:,:]   \n",
    "        flaw_size = [np.sum(bump),np.sum(dent),np.sum(dot)]\n",
    "        names = [\"bump\",\"dent\",\"dot\"]\n",
    "\n",
    "        if not np.sum(flaw_size):\n",
    "            self.num_null_mask += 1\n",
    "            #print(\"==== no flaw for image %d ====\"%i)\n",
    "\n",
    "        if not return_mask:\n",
    "            chips= []\n",
    "            for j in range(3):\n",
    "                if flaw_size[j] >0:\n",
    "                    chips.append(names[j])\n",
    "                    self.all_flaw_size.append(flaw_size[j])\n",
    "            return chips\n",
    "        else:\n",
    "            masks = []\n",
    "            for j in range(3):\n",
    "                if flaw_size[j] >0:\n",
    "                    masks.append(mask_full[j+1,:,:])\n",
    "            if not np.array(masks).shape[0]:\n",
    "                return np.array(masks)\n",
    "            else:\n",
    "                masks = np.array(masks)\n",
    "                masks = masks.reshape((-1,224,224))\n",
    "                masks_reshape = np.zeros((224,224,masks.shape[0]))\n",
    "                #assert masks.shape[1:] in [(224,224)]\n",
    "                for i in range(224):\n",
    "                    for j in range(224):\n",
    "                        masks_reshape[i,j,:] = masks[:,i,j]  \n",
    "                return masks_reshape\n",
    "                      \n",
    "        \n",
    "    def load_image(self,image_id):\n",
    "        info = self.image_info[image_id]\n",
    "        \n",
    "        #print(\"check load_image image_id: \",image_id)\n",
    "        \n",
    "        if self.subset == \"train\":\n",
    "            image_full = train_images[image_id,:].reshape((6,224,224))\n",
    "        else:\n",
    "            image_full = test_images[image_id,:].reshape((6,224,224))\n",
    "            \n",
    "        image_full_reshape = np.zeros((224,224,6))\n",
    "        for i in range(224):\n",
    "            for j in range(224):\n",
    "                image_full_reshape[i,j,:] = image_full[:,i,j]  \n",
    "        image = image_full_reshape\n",
    "        return image\n",
    "\n",
    "    def image_reference(self, image_id):\n",
    "        info = self.image_info[image_id]\n",
    "        if info[\"source\"] == \"chips\":\n",
    "            return info[\"chips\"]\n",
    "        else:\n",
    "            super(self.__class__).image_reference(self, image_id)\n",
    "\n",
    "    def load_mask(self, image_id):\n",
    "        \"\"\"Generate instance masks for shapes of the given image ID.\n",
    "        \"\"\"\n",
    "        info = self.image_info[image_id]\n",
    "        chips = info['chips']\n",
    "        mask = self.mask_prepare(image_id,True)\n",
    "        \n",
    "        class_ids = np.array([self.class_names.index(s) for s in chips])\n",
    "        return mask.astype(np.bool), class_ids.astype(np.int32)\n",
    "\n",
    "    def info_display(self,count):\n",
    "        #flaw_size info\n",
    "        max_flaw_size = np.max(self.all_flaw_size)\n",
    "        min_flaw_size = np.min(self.all_flaw_size)\n",
    "        mean_flaw_size = np.mean(self.all_flaw_size)\n",
    "        \n",
    "        #flaw_kind_info\n",
    "        bump_num,dent_num,dot_num = 0,0,0\n",
    "        for i in range(count):\n",
    "            chips = self.image_info[i]['chips']\n",
    "            if \"bump\" in chips:\n",
    "                bump_num +=1\n",
    "            if \"dent\" in chips:\n",
    "                dent_num +=1\n",
    "            if \"dot\" in chips:\n",
    "                dot_num += 1    \n",
    "                \n",
    "        #display all info\n",
    "        print(\"**** info of %s dataset : ***  \"%self.subset)\n",
    "        print(\"  ratio of null mask images:%d %%\"%(self.num_null_mask/count*100))\n",
    "        print(\"  max flaw size:\",max_flaw_size)\n",
    "        print(\"  min flaw size:\",min_flaw_size)\n",
    "        print(\"  mean flaw size:\",mean_flaw_size)\n",
    "        print(\"  bump ratio:%d %%\"%(bump_num/count*100))\n",
    "        print(\"  dent ratio:%d %%\"%(dent_num/count*100))\n",
    "        print(\"  dot ratio:%d %%\"%(dot_num/count*100))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**** info of test dataset : ***  \n",
      "  ratio of null mask images:0 %\n",
      "  max flaw size: 96\n",
      "  min flaw size: 2\n",
      "  mean flaw size: 29.385158254918736\n",
      "  bump ratio:29 %\n",
      "  dent ratio:41 %\n",
      "  dot ratio:29 %\n"
     ]
    }
   ],
   "source": [
    "# Testing dataset\n",
    "dataset_test = ChipsDataset()\n",
    "dataset_test.load_chips(NUM_TEST,\"test\")\n",
    "dataset_test.prepare()\n",
    "dataset_test.info_display(NUM_TEST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**** info of train dataset : ***  \n",
      "  ratio of null mask images:0 %\n",
      "  max flaw size: 102\n",
      "  min flaw size: 2\n",
      "  mean flaw size: 29.65715769880092\n",
      "  bump ratio:28 %\n",
      "  dent ratio:42 %\n",
      "  dot ratio:28 %\n"
     ]
    }
   ],
   "source": [
    "#Traning dataset\n",
    "#cost several mins \n",
    "dataset_train = ChipsDataset()\n",
    "dataset_train.load_chips(NUM_TRAIN,\"train\")\n",
    "dataset_train.prepare()\n",
    "dataset_train.info_display(NUM_TRAIN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ceate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create model in training mode\n",
    "import os\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "#os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
    "\n",
    "model = modellib.MaskRCNN(mode=\"training\", config=config,\n",
    "                          model_dir=MODEL_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model_path = \"/mnt/sh_flex_storage/malu/venv/CHIPS_MRCNN/logs_six/mask_rcnn_chips_0820.h5\"\n",
    "# Load the last model you trained and continue training\n",
    "model.load_weights(model_path, by_name=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training\n",
    "\n",
    "Train in two stages:\n",
    "1. Only the heads. Here we're freezing all the backbone layers and training only the randomly initialized layers (i.e. the ones that we didn't use pre-trained weights from MS COCO). To train only the head layers, pass `layers='heads'` to the `train()` function.\n",
    "\n",
    "2. Fine-tune all layers. For this simple example it's not necessary, but we're including it to show the process. Simply pass `layers=\"all` to train all layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Starting at epoch 0. LR=0.001\n",
      "\n",
      "Checkpoint Path: /mnt/sh_flex_storage/malu/venv/CHIPS_MRCNN/logs_six/chips20180831T1005/mask_rcnn_chips_{epoch:04d}.h5\n",
      "Selecting layers to train\n",
      "conv1                  (Conv2D)\n",
      "bn_conv1               (BatchNorm)\n",
      "res2a_branch2a         (Conv2D)\n",
      "bn2a_branch2a          (BatchNorm)\n",
      "res2a_branch2b         (Conv2D)\n",
      "bn2a_branch2b          (BatchNorm)\n",
      "res2a_branch2c         (Conv2D)\n",
      "res2a_branch1          (Conv2D)\n",
      "bn2a_branch2c          (BatchNorm)\n",
      "bn2a_branch1           (BatchNorm)\n",
      "res2b_branch2a         (Conv2D)\n",
      "bn2b_branch2a          (BatchNorm)\n",
      "res2b_branch2b         (Conv2D)\n",
      "bn2b_branch2b          (BatchNorm)\n",
      "res2b_branch2c         (Conv2D)\n",
      "bn2b_branch2c          (BatchNorm)\n",
      "res2c_branch2a         (Conv2D)\n",
      "bn2c_branch2a          (BatchNorm)\n",
      "res2c_branch2b         (Conv2D)\n",
      "bn2c_branch2b          (BatchNorm)\n",
      "res2c_branch2c         (Conv2D)\n",
      "bn2c_branch2c          (BatchNorm)\n",
      "res3a_branch2a         (Conv2D)\n",
      "bn3a_branch2a          (BatchNorm)\n",
      "res3a_branch2b         (Conv2D)\n",
      "bn3a_branch2b          (BatchNorm)\n",
      "res3a_branch2c         (Conv2D)\n",
      "res3a_branch1          (Conv2D)\n",
      "bn3a_branch2c          (BatchNorm)\n",
      "bn3a_branch1           (BatchNorm)\n",
      "res3b_branch2a         (Conv2D)\n",
      "bn3b_branch2a          (BatchNorm)\n",
      "res3b_branch2b         (Conv2D)\n",
      "bn3b_branch2b          (BatchNorm)\n",
      "res3b_branch2c         (Conv2D)\n",
      "bn3b_branch2c          (BatchNorm)\n",
      "res3c_branch2a         (Conv2D)\n",
      "bn3c_branch2a          (BatchNorm)\n",
      "res3c_branch2b         (Conv2D)\n",
      "bn3c_branch2b          (BatchNorm)\n",
      "res3c_branch2c         (Conv2D)\n",
      "bn3c_branch2c          (BatchNorm)\n",
      "res3d_branch2a         (Conv2D)\n",
      "bn3d_branch2a          (BatchNorm)\n",
      "res3d_branch2b         (Conv2D)\n",
      "bn3d_branch2b          (BatchNorm)\n",
      "res3d_branch2c         (Conv2D)\n",
      "bn3d_branch2c          (BatchNorm)\n",
      "res4a_branch2a         (Conv2D)\n",
      "bn4a_branch2a          (BatchNorm)\n",
      "res4a_branch2b         (Conv2D)\n",
      "bn4a_branch2b          (BatchNorm)\n",
      "res4a_branch2c         (Conv2D)\n",
      "res4a_branch1          (Conv2D)\n",
      "bn4a_branch2c          (BatchNorm)\n",
      "bn4a_branch1           (BatchNorm)\n",
      "res4b_branch2a         (Conv2D)\n",
      "bn4b_branch2a          (BatchNorm)\n",
      "res4b_branch2b         (Conv2D)\n",
      "bn4b_branch2b          (BatchNorm)\n",
      "res4b_branch2c         (Conv2D)\n",
      "bn4b_branch2c          (BatchNorm)\n",
      "res4c_branch2a         (Conv2D)\n",
      "bn4c_branch2a          (BatchNorm)\n",
      "res4c_branch2b         (Conv2D)\n",
      "bn4c_branch2b          (BatchNorm)\n",
      "res4c_branch2c         (Conv2D)\n",
      "bn4c_branch2c          (BatchNorm)\n",
      "res4d_branch2a         (Conv2D)\n",
      "bn4d_branch2a          (BatchNorm)\n",
      "res4d_branch2b         (Conv2D)\n",
      "bn4d_branch2b          (BatchNorm)\n",
      "res4d_branch2c         (Conv2D)\n",
      "bn4d_branch2c          (BatchNorm)\n",
      "res4e_branch2a         (Conv2D)\n",
      "bn4e_branch2a          (BatchNorm)\n",
      "res4e_branch2b         (Conv2D)\n",
      "bn4e_branch2b          (BatchNorm)\n",
      "res4e_branch2c         (Conv2D)\n",
      "bn4e_branch2c          (BatchNorm)\n",
      "res4f_branch2a         (Conv2D)\n",
      "bn4f_branch2a          (BatchNorm)\n",
      "res4f_branch2b         (Conv2D)\n",
      "bn4f_branch2b          (BatchNorm)\n",
      "res4f_branch2c         (Conv2D)\n",
      "bn4f_branch2c          (BatchNorm)\n",
      "res5a_branch2a         (Conv2D)\n",
      "bn5a_branch2a          (BatchNorm)\n",
      "res5a_branch2b         (Conv2D)\n",
      "bn5a_branch2b          (BatchNorm)\n",
      "res5a_branch2c         (Conv2D)\n",
      "res5a_branch1          (Conv2D)\n",
      "bn5a_branch2c          (BatchNorm)\n",
      "bn5a_branch1           (BatchNorm)\n",
      "res5b_branch2a         (Conv2D)\n",
      "bn5b_branch2a          (BatchNorm)\n",
      "res5b_branch2b         (Conv2D)\n",
      "bn5b_branch2b          (BatchNorm)\n",
      "res5b_branch2c         (Conv2D)\n",
      "bn5b_branch2c          (BatchNorm)\n",
      "res5c_branch2a         (Conv2D)\n",
      "bn5c_branch2a          (BatchNorm)\n",
      "res5c_branch2b         (Conv2D)\n",
      "bn5c_branch2b          (BatchNorm)\n",
      "res5c_branch2c         (Conv2D)\n",
      "bn5c_branch2c          (BatchNorm)\n",
      "fpn_c5p5               (Conv2D)\n",
      "fpn_c4p4               (Conv2D)\n",
      "fpn_c3p3               (Conv2D)\n",
      "fpn_c2p2               (Conv2D)\n",
      "fpn_p5                 (Conv2D)\n",
      "fpn_p2                 (Conv2D)\n",
      "fpn_p3                 (Conv2D)\n",
      "fpn_p4                 (Conv2D)\n",
      "In model:  rpn_model\n",
      "    rpn_conv_shared        (Conv2D)\n",
      "    rpn_class_raw          (Conv2D)\n",
      "    rpn_bbox_pred          (Conv2D)\n",
      "mrcnn_mask_conv1       (TimeDistributed)\n",
      "mrcnn_mask_bn1         (TimeDistributed)\n",
      "mrcnn_mask_conv2       (TimeDistributed)\n",
      "mrcnn_mask_bn2         (TimeDistributed)\n",
      "mrcnn_class_conv1      (TimeDistributed)\n",
      "mrcnn_class_bn1        (TimeDistributed)\n",
      "mrcnn_mask_conv3       (TimeDistributed)\n",
      "mrcnn_mask_bn3         (TimeDistributed)\n",
      "mrcnn_class_conv2      (TimeDistributed)\n",
      "mrcnn_class_bn2        (TimeDistributed)\n",
      "mrcnn_mask_conv4       (TimeDistributed)\n",
      "mrcnn_mask_bn4         (TimeDistributed)\n",
      "mrcnn_bbox_fc          (TimeDistributed)\n",
      "mrcnn_mask_deconv      (TimeDistributed)\n",
      "mrcnn_class_logits     (TimeDistributed)\n",
      "mrcnn_mask             (TimeDistributed)\n",
      "Epoch 1/300\n",
      "100/100 [==============================] - 82s 821ms/step - loss: 1.7254 - rpn_class_loss: 0.0273 - rpn_bbox_loss: 1.3208 - mrcnn_class_loss: 0.0255 - mrcnn_bbox_loss: 0.2072 - mrcnn_mask_loss: 0.1446 - val_loss: 1.4007 - val_rpn_class_loss: 0.0276 - val_rpn_bbox_loss: 1.3688 - val_mrcnn_class_loss: 0.0041 - val_mrcnn_bbox_loss: 6.9701e-05 - val_mrcnn_mask_loss: 1.4203e-04\n",
      "Epoch 2/300\n",
      "100/100 [==============================] - 41s 406ms/step - loss: 1.8440 - rpn_class_loss: 0.0288 - rpn_bbox_loss: 1.3580 - mrcnn_class_loss: 0.0261 - mrcnn_bbox_loss: 0.2250 - mrcnn_mask_loss: 0.2060 - val_loss: 1.4472 - val_rpn_class_loss: 0.0314 - val_rpn_bbox_loss: 1.1963 - val_mrcnn_class_loss: 0.0465 - val_mrcnn_bbox_loss: 0.0906 - val_mrcnn_mask_loss: 0.0824\n",
      "Epoch 3/300\n",
      "100/100 [==============================] - 42s 421ms/step - loss: 1.7875 - rpn_class_loss: 0.0267 - rpn_bbox_loss: 1.2899 - mrcnn_class_loss: 0.0271 - mrcnn_bbox_loss: 0.2343 - mrcnn_mask_loss: 0.2094 - val_loss: 1.7736 - val_rpn_class_loss: 0.0242 - val_rpn_bbox_loss: 1.1442 - val_mrcnn_class_loss: 0.0635 - val_mrcnn_bbox_loss: 0.3148 - val_mrcnn_mask_loss: 0.2268\n",
      "Epoch 4/300\n",
      "100/100 [==============================] - 42s 416ms/step - loss: 1.8689 - rpn_class_loss: 0.0267 - rpn_bbox_loss: 1.2676 - mrcnn_class_loss: 0.0281 - mrcnn_bbox_loss: 0.2808 - mrcnn_mask_loss: 0.2656 - val_loss: 1.4050 - val_rpn_class_loss: 0.0252 - val_rpn_bbox_loss: 1.1373 - val_mrcnn_class_loss: 0.0260 - val_mrcnn_bbox_loss: 0.1127 - val_mrcnn_mask_loss: 0.1037\n",
      "Epoch 5/300\n",
      "100/100 [==============================] - 41s 409ms/step - loss: 1.9070 - rpn_class_loss: 0.0273 - rpn_bbox_loss: 1.2957 - mrcnn_class_loss: 0.0300 - mrcnn_bbox_loss: 0.3057 - mrcnn_mask_loss: 0.2483 - val_loss: 1.8616 - val_rpn_class_loss: 0.0329 - val_rpn_bbox_loss: 1.2196 - val_mrcnn_class_loss: 0.0312 - val_mrcnn_bbox_loss: 0.3450 - val_mrcnn_mask_loss: 0.2328\n",
      "Epoch 6/300\n",
      "100/100 [==============================] - 41s 413ms/step - loss: 1.8637 - rpn_class_loss: 0.0273 - rpn_bbox_loss: 1.2732 - mrcnn_class_loss: 0.0281 - mrcnn_bbox_loss: 0.2833 - mrcnn_mask_loss: 0.2517 - val_loss: 1.5523 - val_rpn_class_loss: 0.0250 - val_rpn_bbox_loss: 1.1173 - val_mrcnn_class_loss: 0.0166 - val_mrcnn_bbox_loss: 0.1502 - val_mrcnn_mask_loss: 0.2433\n",
      "Epoch 7/300\n",
      "100/100 [==============================] - 42s 420ms/step - loss: 1.7747 - rpn_class_loss: 0.0263 - rpn_bbox_loss: 1.2830 - mrcnn_class_loss: 0.0213 - mrcnn_bbox_loss: 0.2303 - mrcnn_mask_loss: 0.2137 - val_loss: 2.2914 - val_rpn_class_loss: 0.0248 - val_rpn_bbox_loss: 1.3229 - val_mrcnn_class_loss: 0.0457 - val_mrcnn_bbox_loss: 0.4975 - val_mrcnn_mask_loss: 0.4005\n",
      "Epoch 8/300\n",
      "100/100 [==============================] - 42s 423ms/step - loss: 1.8386 - rpn_class_loss: 0.0266 - rpn_bbox_loss: 1.3091 - mrcnn_class_loss: 0.0215 - mrcnn_bbox_loss: 0.2538 - mrcnn_mask_loss: 0.2276 - val_loss: 1.7056 - val_rpn_class_loss: 0.0271 - val_rpn_bbox_loss: 1.1734 - val_mrcnn_class_loss: 0.0164 - val_mrcnn_bbox_loss: 0.2724 - val_mrcnn_mask_loss: 0.2163\n",
      "Epoch 9/300\n",
      "100/100 [==============================] - 42s 421ms/step - loss: 1.7479 - rpn_class_loss: 0.0262 - rpn_bbox_loss: 1.2406 - mrcnn_class_loss: 0.0247 - mrcnn_bbox_loss: 0.2370 - mrcnn_mask_loss: 0.2194 - val_loss: 2.0702 - val_rpn_class_loss: 0.0255 - val_rpn_bbox_loss: 1.2807 - val_mrcnn_class_loss: 0.0262 - val_mrcnn_bbox_loss: 0.3455 - val_mrcnn_mask_loss: 0.3923\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10/300\n",
      "100/100 [==============================] - 41s 412ms/step - loss: 1.7226 - rpn_class_loss: 0.0268 - rpn_bbox_loss: 1.2279 - mrcnn_class_loss: 0.0249 - mrcnn_bbox_loss: 0.2231 - mrcnn_mask_loss: 0.2200 - val_loss: 1.6146 - val_rpn_class_loss: 0.0263 - val_rpn_bbox_loss: 1.2511 - val_mrcnn_class_loss: 0.0332 - val_mrcnn_bbox_loss: 0.1669 - val_mrcnn_mask_loss: 0.1372\n",
      "Epoch 11/300\n",
      "100/100 [==============================] - 42s 416ms/step - loss: 1.9266 - rpn_class_loss: 0.0259 - rpn_bbox_loss: 1.2678 - mrcnn_class_loss: 0.0251 - mrcnn_bbox_loss: 0.3187 - mrcnn_mask_loss: 0.2891 - val_loss: 1.5682 - val_rpn_class_loss: 0.0231 - val_rpn_bbox_loss: 1.0741 - val_mrcnn_class_loss: 0.0554 - val_mrcnn_bbox_loss: 0.2173 - val_mrcnn_mask_loss: 0.1983\n",
      "Epoch 12/300\n",
      "100/100 [==============================] - 42s 420ms/step - loss: 1.8857 - rpn_class_loss: 0.0263 - rpn_bbox_loss: 1.2492 - mrcnn_class_loss: 0.0289 - mrcnn_bbox_loss: 0.2992 - mrcnn_mask_loss: 0.2821 - val_loss: 1.6017 - val_rpn_class_loss: 0.0262 - val_rpn_bbox_loss: 1.4158 - val_mrcnn_class_loss: 0.0115 - val_mrcnn_bbox_loss: 0.0456 - val_mrcnn_mask_loss: 0.1027\n",
      "Epoch 13/300\n",
      " 34/100 [=========>....................] - ETA: 26s - loss: 1.6585 - rpn_class_loss: 0.0260 - rpn_bbox_loss: 1.2322 - mrcnn_class_loss: 0.0166 - mrcnn_bbox_loss: 0.1795 - mrcnn_mask_loss: 0.2041"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process Process-76:\n",
      "Process Process-70:\n",
      "Process Process-78:\n",
      "Process Process-72:\n",
      "Process Process-73:\n",
      "Process Process-80:\n",
      "Process Process-83:\n",
      "Process Process-75:\n",
      "Process Process-79:\n",
      "Process Process-71:\n",
      "Process Process-82:\n",
      "Process Process-77:\n",
      "Process Process-85:\n",
      "Process Process-74:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 680, in _data_generator_task\n",
      "    time.sleep(self.wait_time)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 680, in _data_generator_task\n",
      "    time.sleep(self.wait_time)\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 678, in _data_generator_task\n",
      "    self.queue.put((True, generator_output))\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 680, in _data_generator_task\n",
      "    time.sleep(self.wait_time)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 680, in _data_generator_task\n",
      "    time.sleep(self.wait_time)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 678, in _data_generator_task\n",
      "    self.queue.put((True, generator_output))\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 680, in _data_generator_task\n",
      "    time.sleep(self.wait_time)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 678, in _data_generator_task\n",
      "    self.queue.put((True, generator_output))\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<string>\", line 2, in put\n",
      "KeyboardInterrupt\n",
      "  File \"<string>\", line 2, in put\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 678, in _data_generator_task\n",
      "    self.queue.put((True, generator_output))\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 678, in _data_generator_task\n",
      "    self.queue.put((True, generator_output))\n",
      "  File \"/usr/lib/python3.5/multiprocessing/managers.py\", line 717, in _callmethod\n",
      "    kind, result = conn.recv()\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 678, in _data_generator_task\n",
      "    self.queue.put((True, generator_output))\n",
      "  File \"/usr/lib/python3.5/multiprocessing/managers.py\", line 717, in _callmethod\n",
      "    kind, result = conn.recv()\n",
      "  File \"<string>\", line 2, in put\n",
      "  File \"<string>\", line 2, in put\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 250, in recv\n",
      "    buf = self._recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 250, in recv\n",
      "    buf = self._recv_bytes()\n",
      "  File \"<string>\", line 2, in put\n",
      "  File \"/usr/lib/python3.5/multiprocessing/managers.py\", line 717, in _callmethod\n",
      "    kind, result = conn.recv()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/managers.py\", line 717, in _callmethod\n",
      "    kind, result = conn.recv()\n",
      "  File \"<string>\", line 2, in put\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 250, in recv\n",
      "    buf = self._recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/managers.py\", line 717, in _callmethod\n",
      "    kind, result = conn.recv()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 250, in recv\n",
      "    buf = self._recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 250, in recv\n",
      "    buf = self._recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 680, in _data_generator_task\n",
      "    time.sleep(self.wait_time)\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 680, in _data_generator_task\n",
      "    time.sleep(self.wait_time)\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/managers.py\", line 717, in _callmethod\n",
      "    kind, result = conn.recv()\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 680, in _data_generator_task\n",
      "    time.sleep(self.wait_time)\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 250, in recv\n",
      "    buf = self._recv_bytes()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n",
      "Process Process-81:\n",
      "Process Process-84:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 678, in _data_generator_task\n",
      "    self.queue.put((True, generator_output))\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"<string>\", line 2, in put\n",
      "  File \"/mnt/sh_flex_storage/malu/venv/lib/python3.5/site-packages/keras/utils/data_utils.py\", line 678, in _data_generator_task\n",
      "    self.queue.put((True, generator_output))\n",
      "  File \"/usr/lib/python3.5/multiprocessing/managers.py\", line 717, in _callmethod\n",
      "    kind, result = conn.recv()\n",
      "  File \"<string>\", line 2, in put\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 250, in recv\n",
      "    buf = self._recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/managers.py\", line 717, in _callmethod\n",
      "    kind, result = conn.recv()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 250, in recv\n",
      "    buf = self._recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-339b46499ebb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m             \u001b[0mlearning_rate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLEARNING_RATE\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m             \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m300\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m             layers='all')\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"time consuming:\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv/CHIPS_MRCNN/mrcnn_chips/model.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, train_dataset, val_dataset, learning_rate, epochs, layers, augmentation, custom_callbacks, no_augmentation_sources)\u001b[0m\n\u001b[1;32m   2348\u001b[0m             \u001b[0mmax_queue_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2349\u001b[0m             \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2350\u001b[0;31m             \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2351\u001b[0m         )\n\u001b[1;32m   2352\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv/lib/python3.5/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[1;32m     90\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   1413\u001b[0m             \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1414\u001b[0m             \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1415\u001b[0;31m             initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1416\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1417\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0minterfaces\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegacy_generator_methods_support\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv/lib/python3.5/site-packages/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m    211\u001b[0m                 outs = model.train_on_batch(x, y,\n\u001b[1;32m    212\u001b[0m                                             \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 213\u001b[0;31m                                             class_weight=class_weight)\n\u001b[0m\u001b[1;32m    214\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m   1213\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1214\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1215\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1216\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0munpack_singleton\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1217\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2664\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2665\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2666\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2667\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2668\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2634\u001b[0m                                 \u001b[0msymbol_vals\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2635\u001b[0m                                 session)\n\u001b[0;32m-> 2636\u001b[0;31m         \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2637\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2638\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1380\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1381\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1382\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1383\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1384\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "# use_multiprocessing=False\n",
    "model.train(dataset_train, dataset_test, \n",
    "            learning_rate=config.LEARNING_RATE, \n",
    "            epochs=300, \n",
    "            layers='all')\n",
    "print(\"time consuming:\",time.time()-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fine tune all layers\n",
    "# Passing layers=\"all\" trains all layers. You can also \n",
    "# pass a regular expression to select which layers to\n",
    "# train by name pattern.\n",
    "model.train(dataset_train, dataset_test, \n",
    "            learning_rate=config.LEARNING_RATE,\n",
    "            epochs=400, \n",
    "            layers=\"heads\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Save weights\n",
    "# Typically not needed because callbacks save after every epoch\n",
    "# Uncomment to save manually\n",
    "model_path = os.path.join(MODEL_DIR, \"mask_rcnn_chips_0820.h5\")\n",
    "model.keras_model.save_weights(model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class InferenceConfig(ChipsConfig):\n",
    "    GPU_COUNT = 1\n",
    "    IMAGES_PER_GPU = 1\n",
    "\n",
    "inference_config = InferenceConfig()\n",
    "\n",
    "# Recreate the model in inference mode\n",
    "model = modellib.MaskRCNN(mode=\"inference\", \n",
    "                          config=inference_config,\n",
    "                          model_dir= MODEL_DIR)\n",
    "\n",
    "# Get path to saved weights\n",
    "# Either set a specific path or find last trained weights\n",
    "#model_path = os.path.join(MODEL_DIR, \"mask_rcnn_chips.h5\")\n",
    "#model_path = model.find_last()\n",
    "\n",
    "# Load trained weights\n",
    "model_path = os.path.join(MODEL_DIR, \"mask_rcnn_chips_0820.h5\")\n",
    "print(\"Loading weights from \", model_path)\n",
    "model.load_weights(model_path, by_name=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Test on a random image\n",
    "image_id = random.choice(dataset_test.image_ids)\n",
    "original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n",
    "    modellib.load_image_gt(dataset_test, inference_config, \n",
    "                           image_id, use_mini_mask=False)\n",
    "\n",
    "log(\"original_image\", original_image)\n",
    "log(\"image_meta\", image_meta)\n",
    "log(\"gt_class_id\", gt_class_id)\n",
    "log(\"gt_bbox\", gt_bbox)\n",
    "log(\"gt_mask\", gt_mask)\n",
    "\n",
    "original_image_display = original_image[:,:,:3]\n",
    "visualize.display_instances(original_image_display, gt_bbox, gt_mask, gt_class_id, \n",
    "                            dataset_train.class_names, figsize=(8, 8),colors=[(0.0, 1.0, 1.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "results = model.detect([original_image], verbose=1)\n",
    "r = results[0]\n",
    "original_image = original_image[:,:,:3]\n",
    "visualize.display_instances(original_image_display, r['rois'], r['masks'], r['class_ids'], \n",
    "                            dataset_test.class_names, r['scores'], [(0.0, 1.0, 1.0)],ax=get_ax())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "r['masks']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation\n",
    "- After 100 epoch :num_null_mask: 21 mAP:  0.5632911392405063\n",
    "- After 200 epoch（head）,150 epoch(all):num_null_mask: 6 mAP:  0.9290780141843973"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "image_ids = np.random.choice(dataset_test.image_ids, 100)\n",
    "APs = []\n",
    "num_null_mask = 0\n",
    "\n",
    "for image_id in image_ids:\n",
    "    # Load image and ground truth data\n",
    "    image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n",
    "        modellib.load_image_gt(dataset_test, inference_config,\n",
    "                               image_id, use_mini_mask=False)\n",
    "    molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0)\n",
    "    # Run object detection\n",
    "    results = model.detect([image], verbose=0)\n",
    "    r = results[0]\n",
    "\n",
    "    # Compute AP\n",
    "    if r['masks'].shape[2]==0:\n",
    "        num_null_mask += 1\n",
    "        continue\n",
    "    else:\n",
    "        AP, precisions, recalls, overlaps =\\\n",
    "            utils.compute_ap(gt_bbox, gt_class_id, gt_mask,\n",
    "                             r[\"rois\"], r[\"class_ids\"], r[\"scores\"], r['masks'],iou_threshold=0.3)\n",
    "    APs.append(AP)\n",
    "\n",
    "print(\"num_null_mask:\",num_null_mask)\n",
    "print(\"mAP: \", np.mean(APs))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
